﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data.SqlClient;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace WordFrequencyCalculator
{
    class Program
    {
        private static string _articlesString = ConfigurationManager.ConnectionStrings["Articles"].ConnectionString;
        private static string _ontologyString = ConfigurationManager.ConnectionStrings["Ontology"].ConnectionString;

        static void Main(string[] args)
        {
            List<string> words = GetWords().ToList();
            int textCount;
            Dictionary<string, int> wordFreq;
            Dictionary<string, int> textFreq;
            int wordCount;
            FillFrequencyDicts(words, out textCount, out wordFreq, out textFreq, out wordCount);

            for (int index = 0; index < words.Count; index++)
            {
                string word = words[index];
                if (wordFreq[word] <= 0) continue;
                double wf = wordFreq[word] * 1.0 / wordCount;
                double tf = textFreq[word] * 1.0 / textCount;

                using(var connection = new SqlConnection(_ontologyString))
                using(var cmd = connection.CreateCommand())
                {
                    cmd.CommandText = "update Words set TextFrequency = @tf, WordFrequency = @wf where word = @word";
                    cmd.Parameters.AddWithValue("word", word);
                    cmd.Parameters.AddWithValue("wf", wf);
                    cmd.Parameters.AddWithValue("tf", tf);

                    connection.Open();
                    cmd.ExecuteNonQuery();
                }

                if (index % 10000 == 0)
                    Console.WriteLine(index + " words saved.");
            }
        }

        private static void FillFrequencyDicts(List<string> words, out int textCount, out Dictionary<string, int> wordFreq, out Dictionary<string, int> textFreq,
                                               out int wordCount)
        {
            textFreq = words.ToDictionary(x => x, x => 0);
            wordFreq = words.ToDictionary(x => x, x => 0);
            textCount = 0;
            wordCount = 0;

            using(var connection = new SqlConnection(_articlesString))
            using(var cmd = connection.CreateCommand())
            {
                cmd.CommandText = "select title, body from Articles";
                connection.Open();
                using(var reader = cmd.ExecuteReader())
                {
                    while (reader.Read())
                    {
                        string title = reader.GetString(0);
                        string body = reader.GetString(1);
                        string text = title + Environment.NewLine + body;
                        string[] textWords = SplitIntoWords(text).Where(x => x != "p").Select(x => x.ToLower()).ToArray();

                        foreach (var word in textWords)
                        {
                            if (wordFreq.ContainsKey(word))
                                wordFreq[word]++;
                            wordCount++;
                        }

                        foreach (var word in textWords.Distinct())
                        {
                            if (textFreq.ContainsKey(word))
                            {
                                textFreq[word]++;
                            }
                        }

                        textCount++;
                        if (textCount % 10000 == 0)
                            Console.WriteLine(textCount + " texts done.");
                    }
                }
            }
        }

        private static string[] SplitIntoWords(string text)
        {
            return Regex.Split(text, @"\W+");
        }

        private static IEnumerable<string> GetWords()
        {
            using(var connection = new SqlConnection(_ontologyString))
            using(var cmd = connection.CreateCommand())
            {
                cmd.CommandText = "select Word from Words";
                connection.Open();
                using (var reader = cmd.ExecuteReader())
                {
                    while (reader.Read())
                    {
                        yield return reader.GetString(0);
                    }
                }
            }
        }
    }
}
